import time
import datetime
import numpy as np
import pandas as pd
%matplotlib inline
import matplotlib.pyplot as plt
import math
import plotly.offline as py
py.init_notebook_mode(connected=False)
import plotly.graph_objs as go
import plotly.tools as tls
import plotly.figure_factory as ff
from plotly import tools
from sklearn.linear_model import LinearRegression
import statsmodels.api as sm
from scipy import stats
The data can be found at https://www.kaggle.com/airbnb/seattle/data
# Importing Data
calendar = pd.read_csv('seattle/calendar.csv')
reviews = pd.read_csv('seattle/reviews.csv')
listings = pd.read_csv('seattle/listings.csv')
calendar.columns
reviews.columns
listings.loc[listings.id.isna()]
calendar.loc[calendar.listing_id.isna()]
calendar.loc[calendar.date.isna()]
listings.columns
# change price from sting to integer
calendar['price']=calendar['price'].replace('[\$,]', '', regex=True).astype(float)
# Change date from string to datetime object
calendar['date']=pd.to_datetime(calendar['date'])
# Change date from string to datetime object
reviews['date']=pd.to_datetime(reviews['date'])
# change price from sting to integer
listings['price']=listings['price'].replace('[\$,]', '', regex=True).astype(float)
listings['weekly_price']=listings['weekly_price'].replace('[\$,]', '', regex=True).astype(float)
listings['monthly_price']=listings['monthly_price'].replace('[\$,]', '', regex=True).astype(float)
Hypothesis: My hypothesis is that when there is a lower amount of available listings, the price would go up.
To test this I will do a simple linear regression to see if the F-statistic is below 0.05. This will indicate that the available listings do indeed have a significant impact on the list price.
Another part needed to reject the null hypothesis is to indicate that when the higher number of available listings will increase the average list price. To see if this hold true, the coefficient of available listings should be positive.
# Calculations necessary to get x and y columns for simple linear regression
Avail_calendarcounts_Daily=calendar.loc[calendar.available=='t'].groupby('date') \
.agg({'price': 'mean', 'listing_id': pd.Series.nunique}).reset_index()
x_variable=[]
y_variable=list(Avail_calendarcounts_Daily.price)
# Putting x variable into the right format
for x in Avail_calendarcounts_Daily.listing_id:
x_variable.append([x])
X = x_variable
y = y_variable
# Fitting and running Simple linear regression
X2 = sm.add_constant(X)
est = sm.OLS(y, X2)
est2 = est.fit()
print(est2.summary())
After doing the simple linear regression we can see that the R-squared statistic is only 0.004, which means the available apartments account for less than a percent of the variance in price. If we look at the F-statistic, we can also see that it is well above 0.05 which means that there is no significant relationship between the available apartments and the average list price. I, therefore, reject the null hypothesis and can say with confidence that in this dataset there is no clear relationship between list price and available apartments.
"""
Calculations for graph:
In order for the graph to look a little better I decided to group the
data on a weekly basis rather than on a daily basis.
I also decided to count unique listings available to include in the scatter plot.
"""
Avail_calendarcounts=calendar.loc[calendar.available=='t'].groupby(pd.Grouper(key='date', freq='W-MON')) \
.agg({'price': 'mean', 'listing_id': pd.Series.nunique}).reset_index()
trace1 = go.Scatter(
x=list(Avail_calendarcounts.date),
y=list(Avail_calendarcounts.price),
name='Price'
)
trace2 = go.Scatter(
x=list(Avail_calendarcounts.date),
y=list(Avail_calendarcounts.listing_id),
name='Available apartments',
yaxis='y2'
)
data = [trace1, trace2]
layout = go.Layout(
title='Price vs available apartments',
xaxis=dict(
title='Date'
),
yaxis=dict(
title='Average Listing Price($)'
),
yaxis2=dict(
title='Apartments Available',
titlefont=dict(
color='rgb(148, 103, 189)'
),
tickfont=dict(
color='rgb(148, 103, 189)'
),
overlaying='y',
side='right'
)
)
fig = go.Figure(data=data, layout=layout)
py.iplot(fig, filename='multiple-axes-double')
To add onto the statistical test I did to look at the relationship, I also wanted some visual confirmation. Looking at the graph above one can see that the price clearly isn't much influenced by the available apartments. In some cases, price and available apartments would increase together but at other times they move in completely the opposite direction.
find it very interesting how the Airbnb community choose to list their apartments. For me, the living situation can be very important and is probably the first thing most people consider when renting. For me, the living situation is mostly what type of building you live in (property type) as well as the type of sharing privileges you have (room type)
"""
For this graph I wanted to have a horisontal box plot graph.
I also wanted to colour the bars based on the average price of the property type.
"""
# Calculations for graph
property_type_listings=listings.groupby('property_type').agg({'id':'count','price':'mean'}) \
.sort_values(by='id',ascending=True).reset_index()
# Calculations for color
labels=['red','orange','blue','green']
property_type_listings['price_color']=pd.cut(property_type_listings.price,4,labels=labels)
# Draw figure
data = [go.Bar(
x=property_type_listings.id,
y=property_type_listings.property_type,
orientation = 'h',
marker=dict(color=property_type_listings['price_color'])
)]
layout = go.Layout(
margin=dict(
l=120,
r=10,
t=140,
b=80
),
showlegend=False,
title='Most common property types',
xaxis= dict(title= 'Amount of listings'),
yaxis= dict(title= 'Property Type')
)
fig = go.Figure(data=data, layout=layout)
py.iplot(fig, filename='horizontal-bar')
# Calculation to see what percentage house and apartment takes
combinedhouseandapartmet=property_type_listings.loc[property_type_listings.property_type.isin(['House','Apartment'])].id.sum()
alltypes=property_type_listings.id.sum()
print(round(100*combinedhouseandapartmet/alltypes,2),'%')
On this graph, one can see that two-thirds of the listings are for an entire home/apartment. Another 30% of the listings are private rooms and shared rooms only occupy 3% of the listings. To get an even greater understanding I think the best would be to look at room type/property type combinations.
# Calculations to draw room type pie chart
room_type_listings=listings.groupby('room_type').agg({'id':'count'}) \
.sort_values(by='id',ascending=True).reset_index()
roomtype = {
"data": [
{
"values": room_type_listings.id,
"labels": room_type_listings.room_type,
"hole": .5,
"type": "pie",
"rotation":120
},
],
"layout": {
"title":"Room Type Comparison",
}
}
py.iplot(roomtype)
On this graph one can see that two thirds of the listings is for an entire home/apartment. Another 30% of the listings is private rooms and shared rooms only occupy 3% of the listings. To get an even greater understanding I think the best would be to look at room type/property type combinations.
"""
For this graph I only wanted to show the top most used living situations
along with the other group to get a better understanding of the data
"""
# Calculations to get values for pie chart
propandroom_type_combined=listings.groupby(['property_type','room_type']).agg({'id':'count'}) \
.sort_values(by='id',ascending=False).reset_index()
# Create a more readable name
propandroom_type_combined['name']=propandroom_type_combined.property_type+' | '+propandroom_type_combined.room_type
# Plotting graph
inclidedinlist=4
values=list(propandroom_type_combined[:inclidedinlist].id)
values.append(propandroom_type_combined[inclidedinlist:].id.sum())
labels=list(propandroom_type_combined[:inclidedinlist].name)
labels.append('Other')
pandr_type = {
"data": [
{
"values": values,
"labels": labels,
"hole": .5,
"type": "pie",
"rotation":225
},
],
"layout": {
"title":"Property and Room Type Combined Comparison (Top 4)",
}
}
py.iplot(pandr_type)
As can be seen on the graph above, the most common listing situation is apartments that are entirely rented out. This accounts for 37% of all the listings. We can see that house listings are almost half and half when it come to room types, with the entire home being just a little bit more. It is pretty interesting to see that houses are more commenly shared by means of private rooms that apartments. This statistic does sound like it could make sense but I was pretty surprised to see that the house split was so simular. These statistics definately gives one a better understanding of the listings in Seattle.
The best way to visually represent this will be to display a histogram, so to accomplish this I will first have to create bins.
"""
Since only available listings has prices, I removed all the rows where listings where not available.
"""
availablelistings=calendar.loc[(calendar.available=='t')]
# Plot 75 bin histogram
data=[go.Histogram(
x=availablelistings.price,
xbins=dict(
size=75
),
opacity=0.85
)]
layout = go.Layout(
title='Bin Size 75',
xaxis=dict(
title='Price Range',
range=[1, 600]
),
yaxis=dict(
title='Listings'
),
bargap=0.1,
bargroupgap=0.1
)
fig = go.Figure(data=data, layout=layout)
py.iplot(fig)
# Plot 50 bin histogram
data=[go.Histogram(
x=availablelistings.price,
xbins=dict(
size=50
),
opacity=0.85
)]
layout = go.Layout(
title='Bin Size 50',
xaxis=dict(
title='Price Range',
range=[1, 600]
),
yaxis=dict(
title='Listings'
),
bargap=0.1,
bargroupgap=0.1
)
fig = go.Figure(data=data, layout=layout)
py.iplot(fig)
# Plot 25 bin histogram
data=[go.Histogram(
x=availablelistings.price,
xbins=dict(
size=25
),
opacity=0.85
)]
layout = go.Layout(
title='Bin Size 25',
xaxis=dict(
title='Price Range',
range=[1, 600]
),
yaxis=dict(
title='Listings'
),
bargap=0.1,
bargroupgap=0.1
)
fig = go.Figure(data=data, layout=layout)
py.iplot(fig)